{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Image Inpainting with GMCNN model.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/tlatkowski/inpainting-gmcnn-keras/blob/master/colab/Image_Inpainting_with_GMCNN_model.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "75pj38GgBMjC",
        "colab_type": "text"
      },
      "source": [
        "# Download the current version of GMCNN pipeline from GitHub"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kqMxYG1O04bj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!git clone https://github.com/tlatkowski/inpainting-gmcnn-keras.git"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3vM1pdrC4v4M",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!ls"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6ylxBANpCsBq",
        "colab_type": "text"
      },
      "source": [
        "# Download and extract NVIDIA's testing mask dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TtiBIw-81p1L",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget http://masc.cs.gmu.edu/wiki/uploads/partialconv/mask.zip\n",
        "!unzip -q mask.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z9ZWmGDzZueM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!ls"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SYup2q5dGH-3",
        "colab_type": "text"
      },
      "source": [
        "# Download and extract dataset with training images (Places356)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QJIjYKU3Z3WO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget http://data.csail.mit.edu/places/places365/val_large.tar\n",
        "!tar -xf val_large.tar\n",
        "!mkdir images \n",
        "!cp -a val_large/ images"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Jv1D6yfgcCJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!ls"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M4yk-v6cDKKy",
        "colab_type": "text"
      },
      "source": [
        "# Install all requirements"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nuaRQHIUgkEt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install -r inpainting-gmcnn-keras/requirements/requirements.txt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qReehr_EGO9k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n",
        "!unzip ngrok-stable-linux-amd64.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z5FivEGbGQDt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "LOG_DIR = './outputs'\n",
        "get_ipython().system_raw(\n",
        "    'tensorboard --logdir {} --host 0.0.0.0 --port 6006 &'\n",
        "    .format(LOG_DIR)\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ueWBJoNnGU46",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "get_ipython().system_raw('./ngrok http 6006 &')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LRJVi6TnGV9W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "! curl -s http://localhost:4040/api/tunnels | python3 -c \\\n",
        "    \"import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5Z5qBefT2vcX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!mkdir config\n",
        "!cp inpainting-gmcnn-keras/config/main_config.ini config"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5BJ2eyDiG1Yi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%writefile config/main_config.ini\n",
        "[TRAINING]\n",
        "WGAN_TRAINING_RATIO = 1\n",
        "NUM_EPOCHS = 5\n",
        "BATCH_SIZE = 4\n",
        "IMG_HEIGHT = 256\n",
        "IMG_WIDTH = 256\n",
        "NUM_CHANNELS = 3\n",
        "LEARNING_RATE = 0.0001\n",
        "SAVE_MODEL_STEPS_PERIOD = 1000\n",
        "\n",
        "[MODEL]\n",
        "ADD_MASK_AS_GENERATOR_INPUT = True\n",
        "GRADIENT_PENALTY_LOSS_WEIGHT = 10\n",
        "ID_MRF_LOSS_WEIGHT = 0.05\n",
        "ADVERSARIAL_LOSS_WEIGHT = 0.001\n",
        "NN_STRETCH_SIGMA = 0.5\n",
        "VGG_16_LAYERS = 3,6,10\n",
        "ID_MRF_STYLE_WEIGHT = 1.0\n",
        "ID_MRF_CONTENT_WEIGHT = 1.0\n",
        "NUM_GAUSSIAN_STEPS = 3\n",
        "GAUSSIAN_KERNEL_SIZE = 32\n",
        "GAUSSIAN_KERNEL_STD = 40.0"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SdfwO-_kS0_N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!ls"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mHa6NzCNGsIF",
        "colab_type": "text"
      },
      "source": [
        "# Train generator with only confidence reconstruction loss for 5 epochs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_yLA-oDHl8qw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!python inpainting-gmcnn-keras/runner.py --train_path images --mask_path mask --experiment_name \"gmcnn256x256\" -warm_up_generator"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7G7cZPlBElkF",
        "colab_type": "text"
      },
      "source": [
        "# Visualize predicted images for specific training steps in warm-up generator mode"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R5Denu3d1VrW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!ls outputs/gmcnn256x256/predicted_pics/warm_up_generator/"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ro0NuAbm1grE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from IPython.display import Image\n",
        "Image('outputs/gmcnn256x256/predicted_pics/warm_up_generator/step_3000.png')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r0j6E6OD-4hV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "Image('outputs/gmcnn256x256/predicted_pics/warm_up_generator/step_5000.png')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z8Z-pEA3G5ze",
        "colab_type": "text"
      },
      "source": [
        "# Full Wasserstein GAN training mode: generator, local and global discriminators"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hKOmg7hIWNvF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%writefile config/main_config.ini\n",
        "[TRAINING]\n",
        "WGAN_TRAINING_RATIO = 5\n",
        "NUM_EPOCHS = 5\n",
        "BATCH_SIZE = 4\n",
        "IMG_HEIGHT = 256\n",
        "IMG_WIDTH = 256\n",
        "NUM_CHANNELS = 3\n",
        "LEARNING_RATE = 0.0002\n",
        "SAVE_MODEL_STEPS_PERIOD = 500\n",
        "\n",
        "[MODEL]\n",
        "ADD_MASK_AS_GENERATOR_INPUT = True\n",
        "GRADIENT_PENALTY_LOSS_WEIGHT = 10\n",
        "ID_MRF_LOSS_WEIGHT = 0.05\n",
        "ADVERSARIAL_LOSS_WEIGHT = 0.0005\n",
        "NN_STRETCH_SIGMA = 0.5\n",
        "VGG_16_LAYERS = 3,6,10\n",
        "ID_MRF_STYLE_WEIGHT = 1.0\n",
        "ID_MRF_CONTENT_WEIGHT = 1.0\n",
        "NUM_GAUSSIAN_STEPS = 3\n",
        "GAUSSIAN_KERNEL_SIZE = 32\n",
        "GAUSSIAN_KERNEL_STD = 40.0"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dUNq-6v5Fjfw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!python inpainting-gmcnn-keras/runner.py --train_path images --mask_path mask -from_weights --experiment_name \"gmcnn256x256\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UrBtvFhM4X3D",
        "colab_type": "text"
      },
      "source": [
        "# Vizualise results of full model training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rkW2zttf4hsM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!ls outputs/gmcnn256x256/predicted_pics/wgan/"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VNdxEky14rUl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "Image('outputs/gmcnn256x256/predicted_pics/wgan/step_1000.png')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xHwsortY4rdv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "Image('outputs/gmcnn256x256/predicted_pics/wgan/step_2000.png')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0A4AkCL_stHb",
        "colab_type": "text"
      },
      "source": [
        "# Create zip file with model results and checkpoints"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pwuzAQM61j4c",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!zip -r outputs.zip outputs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v5SmLkQmDq_A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ls"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6l9xPZjVPTzt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!rm -rf outputs/"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "taoxlCBuRICF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!python inpainting-gmcnn-keras/runner.py --train_path images --mask_path mask -warm_up_generator -from_weights"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mdqxD2T-s_xB",
        "colab_type": "text"
      },
      "source": [
        "In order to download model result go to: Files -> content, right click on outputs.zip -> Download"
      ]
    }
  ]
}